{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z4KG6YJCdOv1",
        "outputId": "b55603e7-5758-4082-ec64-d056906ba7d4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.2.2-py3-none-any.whl (346 kB)\n",
            "\u001b[K     |████████████████████████████████| 346 kB 14.2 MB/s \n",
            "\u001b[?25hCollecting transformers\n",
            "  Downloading transformers-4.19.2-py3-none-any.whl (4.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.2 MB 65.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[K     |████████████████████████████████| 212 kB 74.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 51.0 MB/s \n",
            "\u001b[?25hCollecting dill<0.3.5\n",
            "  Downloading dill-0.3.4-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[K     |████████████████████████████████| 86 kB 6.6 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<1.0.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.7.0-py3-none-any.whl (86 kB)\n",
            "\u001b[K     |████████████████████████████████| 86 kB 6.7 MB/s \n",
            "\u001b[?25hCollecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.21.6)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.64.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.11.3)\n",
            "Collecting fsspec[http]>=2021.05.0\n",
            "  Downloading fsspec-2022.5.0-py3-none-any.whl (140 kB)\n",
            "\u001b[K     |████████████████████████████████| 140 kB 75.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 73.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.2.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.7.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.9)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2022.5.18.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 75.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 54.1 MB/s \n",
            "\u001b[?25hCollecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 65.3 MB/s \n",
            "\u001b[?25hCollecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (144 kB)\n",
            "\u001b[K     |████████████████████████████████| 144 kB 75.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.12)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.4.0)\n",
            "Collecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (94 kB)\n",
            "\u001b[K     |████████████████████████████████| 94 kB 4.7 MB/s \n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.8.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: multidict, frozenlist, yarl, urllib3, asynctest, async-timeout, aiosignal, pyyaml, fsspec, dill, aiohttp, xxhash, tokenizers, responses, huggingface-hub, transformers, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Attempting uninstall: dill\n",
            "    Found existing installation: dill 0.3.5.1\n",
            "    Uninstalling dill-0.3.5.1:\n",
            "      Successfully uninstalled dill-0.3.5.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 asynctest-0.13.0 datasets-2.2.2 dill-0.3.4 frozenlist-1.3.0 fsspec-2022.5.0 huggingface-hub-0.7.0 multidict-6.0.2 pyyaml-6.0 responses-0.18.0 tokenizers-0.12.1 transformers-4.19.2 urllib3-1.25.11 xxhash-3.0.0 yarl-1.7.2\n"
          ]
        }
      ],
      "source": [
        "!pip install datasets transformers"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 train_model.py -model=bert-base-cased -train_set=train_subtask1.csv -test_set=CTB_forCASE_rsampled.csv"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Bb1ZeGLI-T0",
        "outputId": "977f1011-48f1-438d-e16f-aab4ff4f7f7c"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using custom data configuration default-f6b83b09e802693d\n",
            "Downloading and preparing dataset csv/default to /root/.cache/huggingface/datasets/csv/default-f6b83b09e802693d/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519...\n",
            "\rDownloading data files:   0% 0/1 [00:00<?, ?it/s]\rDownloading data files: 100% 1/1 [00:00<00:00, 6413.31it/s]\n",
            "\rExtracting data files:   0% 0/1 [00:00<?, ?it/s]\rExtracting data files: 100% 1/1 [00:00<00:00, 586.45it/s]\n",
            "Dataset csv downloaded and prepared to /root/.cache/huggingface/datasets/csv/default-f6b83b09e802693d/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519. Subsequent calls will reuse this data.\n",
            "100% 1/1 [00:00<00:00, 517.18it/s]\n",
            "Using custom data configuration default-f62dd2449dbba751\n",
            "Downloading and preparing dataset csv/default to /root/.cache/huggingface/datasets/csv/default-f62dd2449dbba751/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519...\n",
            "Downloading data files: 100% 1/1 [00:00<00:00, 7516.67it/s]\n",
            "Extracting data files: 100% 1/1 [00:00<00:00, 925.49it/s]\n",
            "Dataset csv downloaded and prepared to /root/.cache/huggingface/datasets/csv/default-f62dd2449dbba751/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519. Subsequent calls will reuse this data.\n",
            "100% 1/1 [00:00<00:00, 869.11it/s]\n",
            "Downloading: 100% 29.0/29.0 [00:00<00:00, 27.6kB/s]\n",
            "Downloading: 100% 570/570 [00:00<00:00, 543kB/s]\n",
            "Downloading: 100% 208k/208k [00:00<00:00, 842kB/s] \n",
            "Downloading: 100% 426k/426k [00:00<00:00, 1.04MB/s]\n",
            "100% 3/3 [00:00<00:00,  3.01ba/s]\n",
            "100% 1/1 [00:00<00:00,  6.32ba/s]\n",
            "Downloading: 100% 416M/416M [00:06<00:00, 70.3MB/s]\n",
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "The following columns in the training set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: agreement, num_votes, sample_set, index, text. If agreement, num_votes, sample_set, index, text are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2925\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 1098\n",
            " 33% 366/1098 [02:41<04:56,  2.47it/s]The following columns in the evaluation set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: text, index. If text, index are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 13.94it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.78it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.14it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.75it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.49it/s]\u001b[A\n",
            " 10% 8/80 [00:01<00:09,  7.31it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.21it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.13it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.07it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  7.04it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  7.02it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  7.00it/s]\u001b[A\n",
            " 19% 15/80 [00:02<00:09,  6.99it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  6.99it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:09,  6.97it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  6.95it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  6.95it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  6.95it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  6.95it/s]\u001b[A\n",
            " 28% 22/80 [00:03<00:08,  6.95it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  6.95it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:08,  6.95it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  6.95it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  6.95it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  6.90it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  6.92it/s]\u001b[A\n",
            " 36% 29/80 [00:04<00:07,  6.94it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  6.94it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:07,  6.94it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  6.95it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  6.94it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  6.89it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  6.91it/s]\u001b[A\n",
            " 45% 36/80 [00:05<00:06,  6.92it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  6.93it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:06,  6.92it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  6.93it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  6.94it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  6.93it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  6.94it/s]\u001b[A\n",
            " 54% 43/80 [00:06<00:05,  6.93it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  6.95it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:05,  6.94it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  6.94it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  6.94it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  6.95it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  6.95it/s]\u001b[A\n",
            " 62% 50/80 [00:07<00:04,  6.91it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  6.93it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:04,  6.94it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  6.94it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  6.94it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  6.95it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  6.95it/s]\u001b[A\n",
            " 71% 57/80 [00:08<00:03,  6.94it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  6.95it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:03,  6.96it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  6.96it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  6.96it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  6.97it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  6.96it/s]\u001b[A\n",
            " 80% 64/80 [00:09<00:02,  6.95it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  6.95it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:02,  6.95it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 89% 71/80 [00:10<00:01,  6.96it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  6.97it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:01,  6.97it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  6.97it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  6.96it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  6.96it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  6.96it/s]\u001b[A\n",
            " 98% 78/80 [00:11<00:00,  6.95it/s]\u001b[A\n",
            " 99% 79/80 [00:11<00:00,  6.96it/s]\u001b[A\n",
            "\n",
            "Downloading builder script: 4.21kB [00:00, 3.55MB/s]       \n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.509928286075592, 'eval_accuracy': 0.7657232704402516, 'eval_runtime': 11.8895, 'eval_samples_per_second': 53.493, 'eval_steps_per_second': 6.729, 'epoch': 1.0}\n",
            " 33% 366/1098 [02:52<04:56,  2.47it/s]\n",
            "100% 80/80 [00:11<00:00,  6.96it/s]\u001b[A\n",
            "{'loss': 0.518, 'learning_rate': 2.7231329690346086e-05, 'epoch': 1.37}\n",
            " 46% 500/1098 [03:51<04:23,  2.27it/s]Saving model checkpoint to bert-base-cased_trainer/checkpoint-500\n",
            "Configuration saved in bert-base-cased_trainer/checkpoint-500/config.json\n",
            "Model weights saved in bert-base-cased_trainer/checkpoint-500/pytorch_model.bin\n",
            " 67% 732/1098 [05:38<02:27,  2.48it/s]The following columns in the evaluation set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: text, index. If text, index are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 13.88it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.76it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.14it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.75it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.48it/s]\u001b[A\n",
            " 10% 8/80 [00:01<00:09,  7.22it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.14it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.08it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.03it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  6.96it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  6.96it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  6.95it/s]\u001b[A\n",
            " 19% 15/80 [00:02<00:09,  6.89it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  6.92it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:09,  6.93it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  6.94it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  6.95it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  6.95it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  6.87it/s]\u001b[A\n",
            " 28% 22/80 [00:03<00:08,  6.82it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  6.85it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:08,  6.88it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  6.91it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  6.92it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  6.94it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  6.95it/s]\u001b[A\n",
            " 36% 29/80 [00:04<00:07,  6.93it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  6.89it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:07,  6.91it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  6.92it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  6.93it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  6.94it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  6.92it/s]\u001b[A\n",
            " 45% 36/80 [00:05<00:06,  6.92it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  6.93it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:06,  6.94it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  6.94it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  6.94it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  6.94it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  6.94it/s]\u001b[A\n",
            " 54% 43/80 [00:06<00:05,  6.95it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  6.95it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:05,  6.95it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  6.92it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  6.93it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  6.94it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  6.93it/s]\u001b[A\n",
            " 62% 50/80 [00:07<00:04,  6.93it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  6.93it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:04,  6.93it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  6.89it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  6.92it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  6.93it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  6.94it/s]\u001b[A\n",
            " 71% 57/80 [00:08<00:03,  6.95it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  6.95it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:03,  6.95it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  6.94it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  6.95it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  6.95it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  6.94it/s]\u001b[A\n",
            " 80% 64/80 [00:09<00:02,  6.94it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  6.95it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:02,  6.95it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  6.95it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 89% 71/80 [00:10<00:01,  6.96it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  6.96it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:01,  6.96it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  6.96it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  6.96it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  6.95it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  6.95it/s]\u001b[A\n",
            " 98% 78/80 [00:11<00:00,  6.94it/s]\u001b[A\n",
            " 99% 79/80 [00:11<00:00,  6.94it/s]\u001b[A\n",
            "{'eval_loss': 0.7581159472465515, 'eval_accuracy': 0.7547169811320755, 'eval_runtime': 12.0002, 'eval_samples_per_second': 52.999, 'eval_steps_per_second': 6.667, 'epoch': 2.0}\n",
            "\n",
            " 67% 732/1098 [05:50<02:27,  2.48it/s]\n",
            "{'loss': 0.258, 'learning_rate': 4.4626593806921675e-06, 'epoch': 2.73}\n",
            " 91% 1000/1098 [07:48<00:43,  2.27it/s]Saving model checkpoint to bert-base-cased_trainer/checkpoint-1000\n",
            "Configuration saved in bert-base-cased_trainer/checkpoint-1000/config.json\n",
            "Model weights saved in bert-base-cased_trainer/checkpoint-1000/pytorch_model.bin\n",
            "100% 1098/1098 [08:35<00:00,  2.49it/s]The following columns in the evaluation set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: text, index. If text, index are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 13.92it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.73it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.12it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.74it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.49it/s]\u001b[A\n",
            " 10% 8/80 [00:01<00:09,  7.33it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.22it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.14it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.06it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  7.03it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  7.00it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  6.99it/s]\u001b[A\n",
            " 19% 15/80 [00:02<00:09,  6.98it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  6.98it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:09,  6.97it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  6.97it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  6.94it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  6.94it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  6.95it/s]\u001b[A\n",
            " 28% 22/80 [00:03<00:08,  6.95it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  6.93it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:08,  6.94it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  6.95it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  6.95it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  6.89it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  6.91it/s]\u001b[A\n",
            " 36% 29/80 [00:04<00:07,  6.93it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  6.94it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:07,  6.95it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  6.95it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  6.95it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  6.92it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  6.93it/s]\u001b[A\n",
            " 45% 36/80 [00:05<00:06,  6.94it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  6.95it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:06,  6.95it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  6.94it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  6.94it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  6.95it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  6.95it/s]\u001b[A\n",
            " 54% 43/80 [00:06<00:05,  6.95it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  6.96it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:05,  6.96it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  6.96it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  6.96it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  6.96it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  6.96it/s]\u001b[A\n",
            " 62% 50/80 [00:07<00:04,  6.94it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  6.95it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:04,  6.96it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  6.95it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  6.96it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  6.94it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  6.93it/s]\u001b[A\n",
            " 71% 57/80 [00:08<00:03,  6.88it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  6.91it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:03,  6.93it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  6.94it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  6.95it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  6.96it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  6.96it/s]\u001b[A\n",
            " 80% 64/80 [00:09<00:02,  6.92it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  6.93it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:02,  6.93it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  6.94it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  6.94it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  6.94it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  6.95it/s]\u001b[A\n",
            " 89% 71/80 [00:10<00:01,  6.95it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  6.96it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:01,  6.93it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  6.94it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  6.91it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  6.93it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  6.94it/s]\u001b[A\n",
            " 98% 78/80 [00:11<00:00,  6.95it/s]\u001b[A\n",
            "                                       \n",
            "\u001b[A{'eval_loss': 1.004334807395935, 'eval_accuracy': 0.7594339622641509, 'eval_runtime': 11.8669, 'eval_samples_per_second': 53.595, 'eval_steps_per_second': 6.741, 'epoch': 3.0}\n",
            "100% 1098/1098 [08:47<00:00,  2.49it/s]\n",
            "100% 80/80 [00:11<00:00,  6.95it/s]\u001b[A\n",
            "                                   \u001b[A\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 527.8591, 'train_samples_per_second': 16.624, 'train_steps_per_second': 2.08, 'train_loss': 0.36328899230679096, 'epoch': 3.0}\n",
            "100% 1098/1098 [08:47<00:00,  2.08it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 train_model.py -model=roberta-base -train_set=train_subtask1.csv -test_set=CTB_forCASE_rsampled.csv"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NtehM_H9KED1",
        "outputId": "3f6bd5fa-b9ab-41fe-ce6e-b826f37503a2"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using custom data configuration default-f6b83b09e802693d\n",
            "Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-f6b83b09e802693d/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519)\n",
            "100% 1/1 [00:00<00:00, 749.52it/s]\n",
            "Using custom data configuration default-f62dd2449dbba751\n",
            "Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-f62dd2449dbba751/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519)\n",
            "100% 1/1 [00:00<00:00, 933.10it/s]\n",
            "Downloading: 100% 481/481 [00:00<00:00, 435kB/s]\n",
            "Downloading: 100% 878k/878k [00:00<00:00, 1.76MB/s]\n",
            "Downloading: 100% 446k/446k [00:00<00:00, 1.09MB/s]\n",
            "Downloading: 100% 1.29M/1.29M [00:00<00:00, 2.65MB/s]\n",
            "100% 3/3 [00:00<00:00,  4.59ba/s]\n",
            "100% 1/1 [00:00<00:00,  4.11ba/s]\n",
            "Downloading: 100% 478M/478M [00:07<00:00, 68.8MB/s]\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.dense.bias', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "The following columns in the training set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: index, num_votes, sample_set, agreement, text. If index, num_votes, sample_set, agreement, text are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2925\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 1098\n",
            " 33% 366/1098 [02:41<04:56,  2.47it/s]The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: index, text. If index, text are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 14.06it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.84it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.18it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.80it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.55it/s]\u001b[A\n",
            " 10% 8/80 [00:00<00:09,  7.38it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.27it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.20it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.14it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  7.08it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  7.06it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  7.05it/s]\u001b[A\n",
            " 19% 15/80 [00:01<00:09,  7.05it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  7.04it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:08,  7.04it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  7.02it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  7.02it/s]\u001b[A\n",
            " 28% 22/80 [00:02<00:08,  7.02it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  7.02it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:08,  6.98it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  7.00it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  7.00it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  7.01it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  7.01it/s]\u001b[A\n",
            " 36% 29/80 [00:03<00:07,  7.02it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  7.02it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:06,  7.02it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  7.02it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  7.02it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  7.01it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  7.01it/s]\u001b[A\n",
            " 45% 36/80 [00:04<00:06,  7.01it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  7.01it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:05,  7.01it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  7.02it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  7.02it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  7.02it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  7.02it/s]\u001b[A\n",
            " 54% 43/80 [00:05<00:05,  6.99it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  7.00it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:05,  6.96it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  6.98it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  6.99it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  7.00it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  7.01it/s]\u001b[A\n",
            " 62% 50/80 [00:06<00:04,  7.01it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  7.02it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:03,  7.02it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  7.02it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  7.01it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  6.99it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  6.99it/s]\u001b[A\n",
            " 71% 57/80 [00:07<00:03,  7.00it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  7.01it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:02,  7.01it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  7.01it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 80% 64/80 [00:08<00:02,  7.01it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  7.01it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:01,  7.01it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  7.01it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  7.01it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 89% 71/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  7.02it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:00,  7.02it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  6.98it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  6.99it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  7.00it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  7.01it/s]\u001b[A\n",
            " 98% 78/80 [00:10<00:00,  7.01it/s]\u001b[A\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.69477379322052, 'eval_accuracy': 0.5, 'eval_runtime': 11.9645, 'eval_samples_per_second': 53.157, 'eval_steps_per_second': 6.686, 'epoch': 1.0}\n",
            " 33% 366/1098 [02:53<04:56,  2.47it/s]\n",
            "100% 80/80 [00:11<00:00,  6.99it/s]\u001b[A\n",
            "{'loss': 0.6932, 'learning_rate': 2.7231329690346086e-05, 'epoch': 1.37}\n",
            " 46% 500/1098 [03:52<04:23,  2.27it/s]Saving model checkpoint to roberta-base_trainer/checkpoint-500\n",
            "Configuration saved in roberta-base_trainer/checkpoint-500/config.json\n",
            "Model weights saved in roberta-base_trainer/checkpoint-500/pytorch_model.bin\n",
            " 67% 732/1098 [05:40<02:28,  2.47it/s]The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: index, text. If index, text are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 14.01it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.85it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.22it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.82it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.52it/s]\u001b[A\n",
            " 10% 8/80 [00:00<00:09,  7.37it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.26it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.19it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.13it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  7.10it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  7.08it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  7.03it/s]\u001b[A\n",
            " 19% 15/80 [00:01<00:09,  7.03it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  7.03it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  7.01it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  7.02it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  7.01it/s]\u001b[A\n",
            " 28% 22/80 [00:02<00:08,  7.02it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  7.02it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:07,  7.02it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  6.96it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  6.96it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  6.97it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  6.99it/s]\u001b[A\n",
            " 36% 29/80 [00:03<00:07,  6.99it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  7.00it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:06,  7.01it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  7.00it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  6.97it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  6.98it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  7.00it/s]\u001b[A\n",
            " 45% 36/80 [00:04<00:06,  7.00it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  7.00it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:05,  7.00it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  7.00it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  6.97it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  6.98it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  6.99it/s]\u001b[A\n",
            " 54% 43/80 [00:06<00:05,  7.00it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  7.01it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:04,  7.00it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  7.01it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  7.02it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  7.02it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  7.02it/s]\u001b[A\n",
            " 62% 50/80 [00:06<00:04,  7.03it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  7.02it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:04,  6.99it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  7.00it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  7.01it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  7.01it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  7.02it/s]\u001b[A\n",
            " 71% 57/80 [00:07<00:03,  7.02it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  7.02it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  7.03it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 80% 64/80 [00:08<00:02,  7.00it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  7.01it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:01,  7.01it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  7.01it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 89% 71/80 [00:09<00:01,  6.99it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  7.00it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:00,  7.01it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  7.01it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  7.02it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  7.02it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  7.02it/s]\u001b[A\n",
            " 98% 78/80 [00:10<00:00,  7.02it/s]\u001b[A\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6954694986343384, 'eval_accuracy': 0.5, 'eval_runtime': 11.7704, 'eval_samples_per_second': 54.034, 'eval_steps_per_second': 6.797, 'epoch': 2.0}\n",
            " 67% 732/1098 [05:52<02:28,  2.47it/s]\n",
            "100% 80/80 [00:11<00:00,  7.02it/s]\u001b[A\n",
            "{'loss': 0.6899, 'learning_rate': 4.4626593806921675e-06, 'epoch': 2.73}\n",
            " 91% 1000/1098 [07:50<00:43,  2.27it/s]Saving model checkpoint to roberta-base_trainer/checkpoint-1000\n",
            "Configuration saved in roberta-base_trainer/checkpoint-1000/config.json\n",
            "Model weights saved in roberta-base_trainer/checkpoint-1000/pytorch_model.bin\n",
            "100% 1098/1098 [08:38<00:00,  2.48it/s]The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: index, text. If index, text are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 14.06it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.85it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.22it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.82it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.48it/s]\u001b[A\n",
            " 10% 8/80 [00:01<00:09,  7.34it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.24it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.18it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.13it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  7.09it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  7.07it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  7.06it/s]\u001b[A\n",
            " 19% 15/80 [00:01<00:09,  7.05it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  7.04it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:08,  7.04it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  7.00it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  7.00it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  7.00it/s]\u001b[A\n",
            " 28% 22/80 [00:02<00:08,  7.00it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  7.00it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:07,  7.00it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  7.00it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  6.97it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  6.98it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  7.00it/s]\u001b[A\n",
            " 36% 29/80 [00:03<00:07,  7.00it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  7.01it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:06,  7.01it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  7.02it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  7.00it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  7.00it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  7.00it/s]\u001b[A\n",
            " 45% 36/80 [00:04<00:06,  7.01it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  7.01it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:05,  7.01it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  7.01it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  7.01it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  7.01it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  6.98it/s]\u001b[A\n",
            " 54% 43/80 [00:05<00:05,  7.00it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  7.00it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:05,  6.98it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  6.99it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  7.00it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  7.01it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  7.01it/s]\u001b[A\n",
            " 62% 50/80 [00:06<00:04,  7.01it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  7.01it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:04,  6.99it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  7.00it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  7.01it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  7.01it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  7.02it/s]\u001b[A\n",
            " 71% 57/80 [00:07<00:03,  7.02it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  7.01it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 80% 64/80 [00:08<00:02,  6.99it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  6.99it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:02,  6.97it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  6.99it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  6.99it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  6.99it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  7.00it/s]\u001b[A\n",
            " 89% 71/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  6.98it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:01,  7.00it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  7.01it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  7.01it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  7.00it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  7.00it/s]\u001b[A\n",
            " 98% 78/80 [00:10<00:00,  6.99it/s]\u001b[A\n",
            "                                       \n",
            "\u001b[A{'eval_loss': 0.6552574038505554, 'eval_accuracy': 0.6163522012578616, 'eval_runtime': 11.8971, 'eval_samples_per_second': 53.458, 'eval_steps_per_second': 6.724, 'epoch': 3.0}\n",
            "100% 1098/1098 [08:50<00:00,  2.48it/s]\n",
            "100% 80/80 [00:11<00:00,  6.99it/s]\u001b[A\n",
            "                                   \u001b[A\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 530.7267, 'train_samples_per_second': 16.534, 'train_steps_per_second': 2.069, 'train_loss': 0.6903718764230419, 'epoch': 3.0}\n",
            "100% 1098/1098 [08:50<00:00,  2.07it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 train_model.py -model=google/electra-base-discriminator -train_set=train_subtask1.csv -test_set=CTB_forCASE_rsampled.csv"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "697Ba0JVKXtD",
        "outputId": "0a729eba-09a6-4408-fdb9-04c235d38ac0"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using custom data configuration default-f6b83b09e802693d\n",
            "Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-f6b83b09e802693d/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519)\n",
            "100% 1/1 [00:00<00:00, 674.00it/s]\n",
            "Using custom data configuration default-f62dd2449dbba751\n",
            "Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-f62dd2449dbba751/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519)\n",
            "100% 1/1 [00:00<00:00, 897.56it/s]\n",
            "Downloading: 100% 27.0/27.0 [00:00<00:00, 23.8kB/s]\n",
            "Downloading: 100% 666/666 [00:00<00:00, 595kB/s]\n",
            "Downloading: 100% 226k/226k [00:00<00:00, 685kB/s]\n",
            "Downloading: 100% 455k/455k [00:00<00:00, 1.10MB/s]\n",
            "100% 3/3 [00:01<00:00,  2.97ba/s]\n",
            "100% 1/1 [00:00<00:00,  2.28ba/s]\n",
            "Downloading: 100% 420M/420M [00:08<00:00, 51.0MB/s]\n",
            "Some weights of the model checkpoint at google/electra-base-discriminator were not used when initializing ElectraForSequenceClassification: ['discriminator_predictions.dense.weight', 'discriminator_predictions.dense_prediction.bias', 'discriminator_predictions.dense_prediction.weight', 'discriminator_predictions.dense.bias']\n",
            "- This IS expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at google/electra-base-discriminator and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "The following columns in the training set don't have a corresponding argument in `ElectraForSequenceClassification.forward` and have been ignored: index, sample_set, agreement, num_votes, text. If index, sample_set, agreement, num_votes, text are not expected by `ElectraForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2925\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 1098\n",
            " 33% 366/1098 [02:41<04:55,  2.48it/s]The following columns in the evaluation set don't have a corresponding argument in `ElectraForSequenceClassification.forward` and have been ignored: text, index. If text, index are not expected by `ElectraForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 13.87it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.77it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.14it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.74it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.50it/s]\u001b[A\n",
            " 10% 8/80 [00:01<00:09,  7.33it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.22it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.15it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.10it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  7.06it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  7.00it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  6.99it/s]\u001b[A\n",
            " 19% 15/80 [00:02<00:09,  6.99it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  6.98it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:09,  6.97it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  6.96it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  6.96it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  6.95it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  6.96it/s]\u001b[A\n",
            " 28% 22/80 [00:03<00:08,  6.96it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  6.96it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:08,  6.96it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  6.97it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  6.97it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  6.97it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  6.97it/s]\u001b[A\n",
            " 36% 29/80 [00:04<00:07,  6.97it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  6.97it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:07,  6.97it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  6.97it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  6.97it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  6.90it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  6.92it/s]\u001b[A\n",
            " 45% 36/80 [00:05<00:06,  6.90it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  6.92it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:06,  6.93it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  6.94it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  6.95it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  6.89it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  6.92it/s]\u001b[A\n",
            " 54% 43/80 [00:06<00:05,  6.91it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  6.93it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:05,  6.94it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  6.95it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  6.95it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  6.96it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  6.96it/s]\u001b[A\n",
            " 62% 50/80 [00:07<00:04,  6.96it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  6.96it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:04,  6.96it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  6.96it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  6.96it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  6.97it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  6.97it/s]\u001b[A\n",
            " 71% 57/80 [00:08<00:03,  6.97it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  6.97it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:03,  6.97it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  6.97it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  6.97it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  6.97it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  6.97it/s]\u001b[A\n",
            " 80% 64/80 [00:09<00:02,  6.95it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  6.95it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:02,  6.96it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  6.97it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  6.97it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  6.97it/s]\u001b[A\n",
            " 89% 71/80 [00:10<00:01,  6.97it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  6.97it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:01,  6.97it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  6.97it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  6.97it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  6.97it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  6.96it/s]\u001b[A\n",
            " 98% 78/80 [00:11<00:00,  6.97it/s]\u001b[A\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.4494921565055847, 'eval_accuracy': 0.8034591194968553, 'eval_runtime': 11.836, 'eval_samples_per_second': 53.734, 'eval_steps_per_second': 6.759, 'epoch': 1.0}\n",
            " 33% 366/1098 [02:52<04:55,  2.48it/s]\n",
            "100% 80/80 [00:11<00:00,  6.97it/s]\u001b[A\n",
            "{'loss': 0.4839, 'learning_rate': 2.7231329690346086e-05, 'epoch': 1.37}\n",
            " 46% 500/1098 [03:51<04:23,  2.27it/s]Saving model checkpoint to google/electra-base-discriminator_trainer/checkpoint-500\n",
            "Configuration saved in google/electra-base-discriminator_trainer/checkpoint-500/config.json\n",
            "Model weights saved in google/electra-base-discriminator_trainer/checkpoint-500/pytorch_model.bin\n",
            " 67% 732/1098 [05:38<02:27,  2.49it/s]The following columns in the evaluation set don't have a corresponding argument in `ElectraForSequenceClassification.forward` and have been ignored: text, index. If text, index are not expected by `ElectraForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 13.93it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.76it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.13it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.74it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.48it/s]\u001b[A\n",
            " 10% 8/80 [00:01<00:09,  7.29it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.14it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.08it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.04it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  7.02it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  6.99it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  6.98it/s]\u001b[A\n",
            " 19% 15/80 [00:02<00:09,  6.97it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  6.88it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:09,  6.86it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  6.89it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  6.89it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  6.92it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  6.93it/s]\u001b[A\n",
            " 28% 22/80 [00:03<00:08,  6.94it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  6.88it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:08,  6.91it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  6.92it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  6.91it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  6.92it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  6.92it/s]\u001b[A\n",
            " 36% 29/80 [00:04<00:07,  6.93it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  6.94it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:07,  6.94it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  6.94it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  6.94it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  6.95it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  6.95it/s]\u001b[A\n",
            " 45% 36/80 [00:05<00:06,  6.95it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  6.95it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:06,  6.95it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  6.95it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  6.96it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  6.95it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  6.91it/s]\u001b[A\n",
            " 54% 43/80 [00:06<00:05,  6.92it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  6.92it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:05,  6.93it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  6.93it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  6.93it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  6.94it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  6.92it/s]\u001b[A\n",
            " 62% 50/80 [00:07<00:04,  6.94it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  6.95it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:04,  6.96it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  6.96it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  6.96it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  6.95it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  6.93it/s]\u001b[A\n",
            " 71% 57/80 [00:08<00:03,  6.93it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  6.94it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:03,  6.93it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  6.95it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  6.95it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  6.96it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  6.96it/s]\u001b[A\n",
            " 80% 64/80 [00:09<00:02,  6.97it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  6.95it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:02,  6.96it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  6.96it/s]\u001b[A\n",
            " 89% 71/80 [00:10<00:01,  6.96it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  6.93it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:01,  6.95it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  6.95it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  6.96it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  6.96it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  6.97it/s]\u001b[A\n",
            " 98% 78/80 [00:11<00:00,  6.97it/s]\u001b[A\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6260124444961548, 'eval_accuracy': 0.8050314465408805, 'eval_runtime': 11.9712, 'eval_samples_per_second': 53.128, 'eval_steps_per_second': 6.683, 'epoch': 2.0}\n",
            " 67% 732/1098 [05:50<02:27,  2.49it/s]\n",
            "100% 80/80 [00:11<00:00,  6.91it/s]\u001b[A\n",
            "{'loss': 0.2681, 'learning_rate': 4.4626593806921675e-06, 'epoch': 2.73}\n",
            " 91% 1000/1098 [07:48<00:43,  2.27it/s]Saving model checkpoint to google/electra-base-discriminator_trainer/checkpoint-1000\n",
            "Configuration saved in google/electra-base-discriminator_trainer/checkpoint-1000/config.json\n",
            "Model weights saved in google/electra-base-discriminator_trainer/checkpoint-1000/pytorch_model.bin\n",
            "100% 1098/1098 [08:36<00:00,  2.48it/s]The following columns in the evaluation set don't have a corresponding argument in `ElectraForSequenceClassification.forward` and have been ignored: text, index. If text, index are not expected by `ElectraForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 13.92it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.76it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.12it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.70it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.47it/s]\u001b[A\n",
            " 10% 8/80 [00:01<00:09,  7.31it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.20it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.12it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.06it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  7.03it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  7.00it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  6.99it/s]\u001b[A\n",
            " 19% 15/80 [00:02<00:09,  6.98it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  6.97it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:09,  6.97it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  6.97it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  6.95it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  6.95it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  6.95it/s]\u001b[A\n",
            " 28% 22/80 [00:03<00:08,  6.94it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  6.95it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:08,  6.95it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  6.96it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  6.95it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  6.91it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  6.92it/s]\u001b[A\n",
            " 36% 29/80 [00:04<00:07,  6.88it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  6.90it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:07,  6.91it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  6.92it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  6.92it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  6.93it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  6.92it/s]\u001b[A\n",
            " 45% 36/80 [00:05<00:06,  6.92it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  6.92it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:06,  6.87it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  6.90it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  6.90it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  6.92it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  6.93it/s]\u001b[A\n",
            " 54% 43/80 [00:06<00:05,  6.93it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  6.93it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:05,  6.89it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  6.91it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  6.92it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  6.93it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  6.94it/s]\u001b[A\n",
            " 62% 50/80 [00:07<00:04,  6.95it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  6.95it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:04,  6.96it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  6.96it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  6.92it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  6.93it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  6.93it/s]\u001b[A\n",
            " 71% 57/80 [00:08<00:03,  6.94it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  6.95it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:03,  6.96it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  6.96it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  6.90it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  6.91it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  6.92it/s]\u001b[A\n",
            " 80% 64/80 [00:09<00:02,  6.92it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  6.94it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:02,  6.94it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  6.95it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  6.94it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  6.94it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  6.92it/s]\u001b[A\n",
            " 89% 71/80 [00:10<00:01,  6.92it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  6.92it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:01,  6.93it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  6.94it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  6.94it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  6.94it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  6.89it/s]\u001b[A\n",
            " 98% 78/80 [00:11<00:00,  6.92it/s]\u001b[A\n",
            "                                       \n",
            "\u001b[A{'eval_loss': 0.7992966175079346, 'eval_accuracy': 0.7987421383647799, 'eval_runtime': 11.8894, 'eval_samples_per_second': 53.493, 'eval_steps_per_second': 6.729, 'epoch': 3.0}\n",
            "100% 1098/1098 [08:48<00:00,  2.48it/s]\n",
            "100% 80/80 [00:11<00:00,  6.93it/s]\u001b[A\n",
            "                                   \u001b[A\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 528.1891, 'train_samples_per_second': 16.613, 'train_steps_per_second': 2.079, 'train_loss': 0.35743926175087964, 'epoch': 3.0}\n",
            "100% 1098/1098 [08:48<00:00,  2.08it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 train_model.py -model=albert-base-v2 -train_set=train_subtask1.csv -test_set=CTB_forCASE_rsampled.csv"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v15241k-KomU",
        "outputId": "03269095-bf38-42eb-ef90-d2225e30f951"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using custom data configuration default-f6b83b09e802693d\n",
            "Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-f6b83b09e802693d/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519)\n",
            "100% 1/1 [00:00<00:00, 679.90it/s]\n",
            "Using custom data configuration default-f62dd2449dbba751\n",
            "Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-f62dd2449dbba751/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519)\n",
            "100% 1/1 [00:00<00:00, 745.65it/s]\n",
            "Downloading: 100% 684/684 [00:00<00:00, 587kB/s]\n",
            "Downloading: 100% 742k/742k [00:00<00:00, 1.78MB/s]\n",
            "Downloading: 100% 1.25M/1.25M [00:00<00:00, 2.57MB/s]\n",
            "100% 3/3 [00:00<00:00,  3.51ba/s]\n",
            "100% 1/1 [00:00<00:00,  3.28ba/s]\n",
            "Downloading: 100% 45.2M/45.2M [00:00<00:00, 74.7MB/s]\n",
            "Some weights of the model checkpoint at albert-base-v2 were not used when initializing AlbertForSequenceClassification: ['predictions.dense.weight', 'predictions.decoder.bias', 'predictions.LayerNorm.weight', 'predictions.dense.bias', 'predictions.bias', 'predictions.LayerNorm.bias', 'predictions.decoder.weight']\n",
            "- This IS expected if you are initializing AlbertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing AlbertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of AlbertForSequenceClassification were not initialized from the model checkpoint at albert-base-v2 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "The following columns in the training set don't have a corresponding argument in `AlbertForSequenceClassification.forward` and have been ignored: sample_set, text, num_votes, agreement, index. If sample_set, text, num_votes, agreement, index are not expected by `AlbertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2925\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 1098\n",
            " 33% 366/1098 [02:45<05:00,  2.43it/s]The following columns in the evaluation set don't have a corresponding argument in `AlbertForSequenceClassification.forward` and have been ignored: index, text. If index, text are not expected by `AlbertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:06, 12.38it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:09,  7.80it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:10,  7.26it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:10,  6.92it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:10,  6.69it/s]\u001b[A\n",
            " 10% 8/80 [00:01<00:11,  6.53it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:11,  6.44it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:10,  6.37it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:10,  6.32it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:10,  6.29it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:10,  6.26it/s]\u001b[A\n",
            " 18% 14/80 [00:02<00:10,  6.25it/s]\u001b[A\n",
            " 19% 15/80 [00:02<00:10,  6.24it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:10,  6.23it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:10,  6.23it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:09,  6.22it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:09,  6.22it/s]\u001b[A\n",
            " 25% 20/80 [00:03<00:09,  6.22it/s]\u001b[A\n",
            " 26% 21/80 [00:03<00:09,  6.21it/s]\u001b[A\n",
            " 28% 22/80 [00:03<00:09,  6.21it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:09,  6.21it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:09,  6.22it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:08,  6.21it/s]\u001b[A\n",
            " 32% 26/80 [00:04<00:08,  6.22it/s]\u001b[A\n",
            " 34% 27/80 [00:04<00:08,  6.21it/s]\u001b[A\n",
            " 35% 28/80 [00:04<00:08,  6.19it/s]\u001b[A\n",
            " 36% 29/80 [00:04<00:08,  6.20it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:08,  6.21it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:07,  6.20it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:07,  6.20it/s]\u001b[A\n",
            " 41% 33/80 [00:05<00:07,  6.21it/s]\u001b[A\n",
            " 42% 34/80 [00:05<00:07,  6.21it/s]\u001b[A\n",
            " 44% 35/80 [00:05<00:07,  6.21it/s]\u001b[A\n",
            " 45% 36/80 [00:05<00:07,  6.21it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  6.21it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:06,  6.21it/s]\u001b[A\n",
            " 49% 39/80 [00:06<00:06,  6.21it/s]\u001b[A\n",
            " 50% 40/80 [00:06<00:06,  6.20it/s]\u001b[A\n",
            " 51% 41/80 [00:06<00:06,  6.15it/s]\u001b[A\n",
            " 52% 42/80 [00:06<00:06,  6.16it/s]\u001b[A\n",
            " 54% 43/80 [00:06<00:05,  6.18it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  6.19it/s]\u001b[A\n",
            " 56% 45/80 [00:07<00:05,  6.20it/s]\u001b[A\n",
            " 57% 46/80 [00:07<00:05,  6.19it/s]\u001b[A\n",
            " 59% 47/80 [00:07<00:05,  6.20it/s]\u001b[A\n",
            " 60% 48/80 [00:07<00:05,  6.20it/s]\u001b[A\n",
            " 61% 49/80 [00:07<00:04,  6.21it/s]\u001b[A\n",
            " 62% 50/80 [00:07<00:04,  6.21it/s]\u001b[A\n",
            " 64% 51/80 [00:08<00:04,  6.20it/s]\u001b[A\n",
            " 65% 52/80 [00:08<00:04,  6.20it/s]\u001b[A\n",
            " 66% 53/80 [00:08<00:04,  6.20it/s]\u001b[A\n",
            " 68% 54/80 [00:08<00:04,  6.21it/s]\u001b[A\n",
            " 69% 55/80 [00:08<00:04,  6.21it/s]\u001b[A\n",
            " 70% 56/80 [00:08<00:03,  6.21it/s]\u001b[A\n",
            " 71% 57/80 [00:09<00:03,  6.21it/s]\u001b[A\n",
            " 72% 58/80 [00:09<00:03,  6.21it/s]\u001b[A\n",
            " 74% 59/80 [00:09<00:03,  6.21it/s]\u001b[A\n",
            " 75% 60/80 [00:09<00:03,  6.21it/s]\u001b[A\n",
            " 76% 61/80 [00:09<00:03,  6.22it/s]\u001b[A\n",
            " 78% 62/80 [00:09<00:02,  6.21it/s]\u001b[A\n",
            " 79% 63/80 [00:09<00:02,  6.21it/s]\u001b[A\n",
            " 80% 64/80 [00:10<00:02,  6.21it/s]\u001b[A\n",
            " 81% 65/80 [00:10<00:02,  6.21it/s]\u001b[A\n",
            " 82% 66/80 [00:10<00:02,  6.22it/s]\u001b[A\n",
            " 84% 67/80 [00:10<00:02,  6.22it/s]\u001b[A\n",
            " 85% 68/80 [00:10<00:01,  6.21it/s]\u001b[A\n",
            " 86% 69/80 [00:10<00:01,  6.21it/s]\u001b[A\n",
            " 88% 70/80 [00:11<00:01,  6.21it/s]\u001b[A\n",
            " 89% 71/80 [00:11<00:01,  6.20it/s]\u001b[A\n",
            " 90% 72/80 [00:11<00:01,  6.20it/s]\u001b[A\n",
            " 91% 73/80 [00:11<00:01,  6.21it/s]\u001b[A\n",
            " 92% 74/80 [00:11<00:00,  6.21it/s]\u001b[A\n",
            " 94% 75/80 [00:11<00:00,  6.22it/s]\u001b[A\n",
            " 95% 76/80 [00:12<00:00,  6.22it/s]\u001b[A\n",
            " 96% 77/80 [00:12<00:00,  6.21it/s]\u001b[A\n",
            " 98% 78/80 [00:12<00:00,  6.15it/s]\u001b[A\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.693155825138092, 'eval_accuracy': 0.5, 'eval_runtime': 13.4154, 'eval_samples_per_second': 47.408, 'eval_steps_per_second': 5.963, 'epoch': 1.0}\n",
            " 33% 366/1098 [02:58<05:00,  2.43it/s]\n",
            "100% 80/80 [00:13<00:00,  6.11it/s]\u001b[A\n",
            "{'loss': 0.7072, 'learning_rate': 2.7231329690346086e-05, 'epoch': 1.37}\n",
            " 46% 500/1098 [03:59<04:31,  2.21it/s]Saving model checkpoint to albert-base-v2_trainer/checkpoint-500\n",
            "Configuration saved in albert-base-v2_trainer/checkpoint-500/config.json\n",
            "Model weights saved in albert-base-v2_trainer/checkpoint-500/pytorch_model.bin\n",
            " 67% 732/1098 [05:43<02:30,  2.43it/s]The following columns in the evaluation set don't have a corresponding argument in `AlbertForSequenceClassification.forward` and have been ignored: index, text. If index, text are not expected by `AlbertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:06, 12.44it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:09,  7.80it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:10,  7.25it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:10,  6.91it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:10,  6.68it/s]\u001b[A\n",
            " 10% 8/80 [00:01<00:11,  6.51it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:11,  6.42it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:11,  6.33it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:10,  6.29it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:10,  6.27it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:10,  6.25it/s]\u001b[A\n",
            " 18% 14/80 [00:02<00:10,  6.24it/s]\u001b[A\n",
            " 19% 15/80 [00:02<00:10,  6.23it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:10,  6.23it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:10,  6.17it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:10,  6.18it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:09,  6.19it/s]\u001b[A\n",
            " 25% 20/80 [00:03<00:09,  6.20it/s]\u001b[A\n",
            " 26% 21/80 [00:03<00:09,  6.20it/s]\u001b[A\n",
            " 28% 22/80 [00:03<00:09,  6.20it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:09,  6.20it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:09,  6.20it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:08,  6.21it/s]\u001b[A\n",
            " 32% 26/80 [00:04<00:08,  6.21it/s]\u001b[A\n",
            " 34% 27/80 [00:04<00:08,  6.21it/s]\u001b[A\n",
            " 35% 28/80 [00:04<00:08,  6.21it/s]\u001b[A\n",
            " 36% 29/80 [00:04<00:08,  6.21it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:08,  6.21it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:07,  6.21it/s]\u001b[A\n",
            " 40% 32/80 [00:05<00:07,  6.21it/s]\u001b[A\n",
            " 41% 33/80 [00:05<00:07,  6.21it/s]\u001b[A\n",
            " 42% 34/80 [00:05<00:07,  6.21it/s]\u001b[A\n",
            " 44% 35/80 [00:05<00:07,  6.20it/s]\u001b[A\n",
            " 45% 36/80 [00:05<00:07,  6.20it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  6.20it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:06,  6.19it/s]\u001b[A\n",
            " 49% 39/80 [00:06<00:06,  6.19it/s]\u001b[A\n",
            " 50% 40/80 [00:06<00:06,  6.19it/s]\u001b[A\n",
            " 51% 41/80 [00:06<00:06,  6.20it/s]\u001b[A\n",
            " 52% 42/80 [00:06<00:06,  6.20it/s]\u001b[A\n",
            " 54% 43/80 [00:06<00:05,  6.21it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  6.21it/s]\u001b[A\n",
            " 56% 45/80 [00:07<00:05,  6.21it/s]\u001b[A\n",
            " 57% 46/80 [00:07<00:05,  6.21it/s]\u001b[A\n",
            " 59% 47/80 [00:07<00:05,  6.21it/s]\u001b[A\n",
            " 60% 48/80 [00:07<00:05,  6.21it/s]\u001b[A\n",
            " 61% 49/80 [00:07<00:04,  6.21it/s]\u001b[A\n",
            " 62% 50/80 [00:07<00:04,  6.21it/s]\u001b[A\n",
            " 64% 51/80 [00:08<00:04,  6.21it/s]\u001b[A\n",
            " 65% 52/80 [00:08<00:04,  6.21it/s]\u001b[A\n",
            " 66% 53/80 [00:08<00:04,  6.21it/s]\u001b[A\n",
            " 68% 54/80 [00:08<00:04,  6.21it/s]\u001b[A\n",
            " 69% 55/80 [00:08<00:04,  6.21it/s]\u001b[A\n",
            " 70% 56/80 [00:08<00:03,  6.21it/s]\u001b[A\n",
            " 71% 57/80 [00:09<00:03,  6.21it/s]\u001b[A\n",
            " 72% 58/80 [00:09<00:03,  6.20it/s]\u001b[A\n",
            " 74% 59/80 [00:09<00:03,  6.20it/s]\u001b[A\n",
            " 75% 60/80 [00:09<00:03,  6.20it/s]\u001b[A\n",
            " 76% 61/80 [00:09<00:03,  6.20it/s]\u001b[A\n",
            " 78% 62/80 [00:09<00:02,  6.21it/s]\u001b[A\n",
            " 79% 63/80 [00:09<00:02,  6.21it/s]\u001b[A\n",
            " 80% 64/80 [00:10<00:02,  6.21it/s]\u001b[A\n",
            " 81% 65/80 [00:10<00:02,  6.21it/s]\u001b[A\n",
            " 82% 66/80 [00:10<00:02,  6.21it/s]\u001b[A\n",
            " 84% 67/80 [00:10<00:02,  6.21it/s]\u001b[A\n",
            " 85% 68/80 [00:10<00:01,  6.20it/s]\u001b[A\n",
            " 86% 69/80 [00:10<00:01,  6.20it/s]\u001b[A\n",
            " 88% 70/80 [00:11<00:01,  6.20it/s]\u001b[A\n",
            " 89% 71/80 [00:11<00:01,  6.21it/s]\u001b[A\n",
            " 90% 72/80 [00:11<00:01,  6.21it/s]\u001b[A\n",
            " 91% 73/80 [00:11<00:01,  6.20it/s]\u001b[A\n",
            " 92% 74/80 [00:11<00:00,  6.20it/s]\u001b[A\n",
            " 94% 75/80 [00:11<00:00,  6.20it/s]\u001b[A\n",
            " 95% 76/80 [00:12<00:00,  6.21it/s]\u001b[A\n",
            " 96% 77/80 [00:12<00:00,  6.21it/s]\u001b[A\n",
            " 98% 78/80 [00:12<00:00,  6.21it/s]\u001b[A\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6996015906333923, 'eval_accuracy': 0.5, 'eval_runtime': 13.2405, 'eval_samples_per_second': 48.034, 'eval_steps_per_second': 6.042, 'epoch': 2.0}\n",
            " 67% 732/1098 [05:57<02:30,  2.43it/s]\n",
            "100% 80/80 [00:13<00:00,  6.21it/s]\u001b[A\n",
            "{'loss': 0.7007, 'learning_rate': 4.4626593806921675e-06, 'epoch': 2.73}\n",
            " 91% 1000/1098 [07:58<00:44,  2.22it/s]Saving model checkpoint to albert-base-v2_trainer/checkpoint-1000\n",
            "Configuration saved in albert-base-v2_trainer/checkpoint-1000/config.json\n",
            "Model weights saved in albert-base-v2_trainer/checkpoint-1000/pytorch_model.bin\n",
            "100% 1098/1098 [08:42<00:00,  2.43it/s]The following columns in the evaluation set don't have a corresponding argument in `AlbertForSequenceClassification.forward` and have been ignored: index, text. If index, text are not expected by `AlbertForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:06, 12.40it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:09,  7.82it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:10,  7.27it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:10,  6.92it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:10,  6.69it/s]\u001b[A\n",
            " 10% 8/80 [00:01<00:11,  6.54it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:11,  6.44it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:10,  6.37it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:10,  6.32it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:10,  6.29it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:10,  6.26it/s]\u001b[A\n",
            " 18% 14/80 [00:02<00:10,  6.25it/s]\u001b[A\n",
            " 19% 15/80 [00:02<00:10,  6.24it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:10,  6.23it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:10,  6.22it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:09,  6.22it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:09,  6.21it/s]\u001b[A\n",
            " 25% 20/80 [00:03<00:09,  6.16it/s]\u001b[A\n",
            " 26% 21/80 [00:03<00:09,  6.17it/s]\u001b[A\n",
            " 28% 22/80 [00:03<00:09,  6.16it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:09,  6.18it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:09,  6.19it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:08,  6.19it/s]\u001b[A\n",
            " 32% 26/80 [00:04<00:08,  6.20it/s]\u001b[A\n",
            " 34% 27/80 [00:04<00:08,  6.20it/s]\u001b[A\n",
            " 35% 28/80 [00:04<00:08,  6.21it/s]\u001b[A\n",
            " 36% 29/80 [00:04<00:08,  6.21it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:08,  6.20it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:07,  6.20it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:07,  6.21it/s]\u001b[A\n",
            " 41% 33/80 [00:05<00:07,  6.21it/s]\u001b[A\n",
            " 42% 34/80 [00:05<00:07,  6.21it/s]\u001b[A\n",
            " 44% 35/80 [00:05<00:07,  6.19it/s]\u001b[A\n",
            " 45% 36/80 [00:05<00:07,  6.19it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  6.20it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:06,  6.20it/s]\u001b[A\n",
            " 49% 39/80 [00:06<00:06,  6.20it/s]\u001b[A\n",
            " 50% 40/80 [00:06<00:06,  6.20it/s]\u001b[A\n",
            " 51% 41/80 [00:06<00:06,  6.21it/s]\u001b[A\n",
            " 52% 42/80 [00:06<00:06,  6.21it/s]\u001b[A\n",
            " 54% 43/80 [00:06<00:05,  6.22it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  6.22it/s]\u001b[A\n",
            " 56% 45/80 [00:07<00:05,  6.22it/s]\u001b[A\n",
            " 57% 46/80 [00:07<00:05,  6.21it/s]\u001b[A\n",
            " 59% 47/80 [00:07<00:05,  6.21it/s]\u001b[A\n",
            " 60% 48/80 [00:07<00:05,  6.22it/s]\u001b[A\n",
            " 61% 49/80 [00:07<00:04,  6.22it/s]\u001b[A\n",
            " 62% 50/80 [00:07<00:04,  6.22it/s]\u001b[A\n",
            " 64% 51/80 [00:08<00:04,  6.22it/s]\u001b[A\n",
            " 65% 52/80 [00:08<00:04,  6.16it/s]\u001b[A\n",
            " 66% 53/80 [00:08<00:04,  6.18it/s]\u001b[A\n",
            " 68% 54/80 [00:08<00:04,  6.19it/s]\u001b[A\n",
            " 69% 55/80 [00:08<00:04,  6.19it/s]\u001b[A\n",
            " 70% 56/80 [00:08<00:03,  6.20it/s]\u001b[A\n",
            " 71% 57/80 [00:09<00:03,  6.19it/s]\u001b[A\n",
            " 72% 58/80 [00:09<00:03,  6.20it/s]\u001b[A\n",
            " 74% 59/80 [00:09<00:03,  6.21it/s]\u001b[A\n",
            " 75% 60/80 [00:09<00:03,  6.21it/s]\u001b[A\n",
            " 76% 61/80 [00:09<00:03,  6.21it/s]\u001b[A\n",
            " 78% 62/80 [00:09<00:02,  6.21it/s]\u001b[A\n",
            " 79% 63/80 [00:09<00:02,  6.21it/s]\u001b[A\n",
            " 80% 64/80 [00:10<00:02,  6.20it/s]\u001b[A\n",
            " 81% 65/80 [00:10<00:02,  6.19it/s]\u001b[A\n",
            " 82% 66/80 [00:10<00:02,  6.20it/s]\u001b[A\n",
            " 84% 67/80 [00:10<00:02,  6.21it/s]\u001b[A\n",
            " 85% 68/80 [00:10<00:01,  6.21it/s]\u001b[A\n",
            " 86% 69/80 [00:10<00:01,  6.20it/s]\u001b[A\n",
            " 88% 70/80 [00:11<00:01,  6.21it/s]\u001b[A\n",
            " 89% 71/80 [00:11<00:01,  6.21it/s]\u001b[A\n",
            " 90% 72/80 [00:11<00:01,  6.21it/s]\u001b[A\n",
            " 91% 73/80 [00:11<00:01,  6.21it/s]\u001b[A\n",
            " 92% 74/80 [00:11<00:00,  6.22it/s]\u001b[A\n",
            " 94% 75/80 [00:11<00:00,  6.22it/s]\u001b[A\n",
            " 95% 76/80 [00:12<00:00,  6.22it/s]\u001b[A\n",
            " 96% 77/80 [00:12<00:00,  6.22it/s]\u001b[A\n",
            " 98% 78/80 [00:12<00:00,  6.22it/s]\u001b[A\n",
            "                                       \n",
            "\u001b[A{'eval_loss': 0.7013487815856934, 'eval_accuracy': 0.5, 'eval_runtime': 13.3424, 'eval_samples_per_second': 47.668, 'eval_steps_per_second': 5.996, 'epoch': 3.0}\n",
            "100% 1098/1098 [08:55<00:00,  2.43it/s]\n",
            "100% 80/80 [00:13<00:00,  6.22it/s]\u001b[A\n",
            "                                   \u001b[A\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 535.8214, 'train_samples_per_second': 16.377, 'train_steps_per_second': 2.049, 'train_loss': 0.7029626981808188, 'epoch': 3.0}\n",
            "100% 1098/1098 [08:55<00:00,  2.05it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 train_model.py -model=xlm-roberta-base -train_set=train_subtask1.csv -test_set=CTB_forCASE_rsampled.csv"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dfNDarIJKwLJ",
        "outputId": "14f244a7-a4e5-4360-c33e-3e87c34d28af"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using custom data configuration default-f6b83b09e802693d\n",
            "Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-f6b83b09e802693d/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519)\n",
            "100% 1/1 [00:00<00:00, 747.38it/s]\n",
            "Using custom data configuration default-f62dd2449dbba751\n",
            "Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-f62dd2449dbba751/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519)\n",
            "100% 1/1 [00:00<00:00, 816.97it/s]\n",
            "Downloading: 100% 615/615 [00:00<00:00, 565kB/s]\n",
            "Downloading: 100% 4.83M/4.83M [00:00<00:00, 6.66MB/s]\n",
            "Downloading: 100% 8.68M/8.68M [00:00<00:00, 10.6MB/s]\n",
            "100% 3/3 [00:00<00:00,  4.58ba/s]\n",
            "100% 1/1 [00:00<00:00,  3.90ba/s]\n",
            "Downloading: 100% 1.04G/1.04G [00:15<00:00, 72.3MB/s]\n",
            "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForSequenceClassification: ['lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'lm_head.dense.weight', 'lm_head.bias', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "- This IS expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of XLMRobertaForSequenceClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "The following columns in the training set don't have a corresponding argument in `XLMRobertaForSequenceClassification.forward` and have been ignored: sample_set, agreement, index, text, num_votes. If sample_set, agreement, index, text, num_votes are not expected by `XLMRobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2925\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 1098\n",
            " 33% 366/1098 [02:51<05:15,  2.32it/s]The following columns in the evaluation set don't have a corresponding argument in `XLMRobertaForSequenceClassification.forward` and have been ignored: index, text. If index, text are not expected by `XLMRobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 14.00it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.85it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.22it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.82it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.54it/s]\u001b[A\n",
            " 10% 8/80 [00:00<00:09,  7.38it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.27it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.20it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.14it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  7.08it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  7.07it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  7.02it/s]\u001b[A\n",
            " 19% 15/80 [00:01<00:09,  7.03it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  7.03it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  7.02it/s]\u001b[A\n",
            " 28% 22/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  7.03it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:07,  7.03it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  7.03it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  6.99it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  7.00it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  7.00it/s]\u001b[A\n",
            " 36% 29/80 [00:03<00:07,  6.97it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  6.97it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:07,  6.98it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  6.99it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  6.96it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  6.98it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  7.00it/s]\u001b[A\n",
            " 45% 36/80 [00:04<00:06,  7.01it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  7.02it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:06,  7.00it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  7.01it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  7.02it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  7.02it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  7.02it/s]\u001b[A\n",
            " 54% 43/80 [00:05<00:05,  6.98it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  7.00it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:04,  7.01it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  7.01it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  7.02it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  7.01it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  7.02it/s]\u001b[A\n",
            " 62% 50/80 [00:06<00:04,  7.01it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  7.02it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:04,  6.97it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  6.98it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  7.00it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  7.00it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  7.01it/s]\u001b[A\n",
            " 71% 57/80 [00:07<00:03,  7.02it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  7.02it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:02,  7.03it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  7.03it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  7.03it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  7.03it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  7.03it/s]\u001b[A\n",
            " 80% 64/80 [00:08<00:02,  7.00it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  7.01it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  7.01it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 89% 71/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  7.02it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:00,  7.02it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  7.03it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  7.03it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  7.02it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  7.03it/s]\u001b[A\n",
            " 98% 78/80 [00:10<00:00,  7.03it/s]\u001b[A\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.693514347076416, 'eval_accuracy': 0.5, 'eval_runtime': 11.7941, 'eval_samples_per_second': 53.925, 'eval_steps_per_second': 6.783, 'epoch': 1.0}\n",
            " 33% 366/1098 [03:03<05:15,  2.32it/s]\n",
            "100% 80/80 [00:11<00:00,  7.02it/s]\u001b[A\n",
            "{'loss': 0.6972, 'learning_rate': 2.7231329690346086e-05, 'epoch': 1.37}\n",
            " 46% 500/1098 [04:05<04:39,  2.14it/s]Saving model checkpoint to xlm-roberta-base_trainer/checkpoint-500\n",
            "Configuration saved in xlm-roberta-base_trainer/checkpoint-500/config.json\n",
            "Model weights saved in xlm-roberta-base_trainer/checkpoint-500/pytorch_model.bin\n",
            " 67% 732/1098 [06:07<02:38,  2.31it/s]The following columns in the evaluation set don't have a corresponding argument in `XLMRobertaForSequenceClassification.forward` and have been ignored: index, text. If index, text are not expected by `XLMRobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 14.08it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.86it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.23it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.84it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.57it/s]\u001b[A\n",
            " 10% 8/80 [00:00<00:09,  7.40it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.29it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.21it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.14it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  7.10it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  7.03it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  7.03it/s]\u001b[A\n",
            " 19% 15/80 [00:01<00:09,  7.03it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  7.03it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  7.04it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  7.04it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  7.00it/s]\u001b[A\n",
            " 28% 22/80 [00:02<00:08,  7.01it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  6.99it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:07,  7.00it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  7.01it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  7.02it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  7.02it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  7.03it/s]\u001b[A\n",
            " 36% 29/80 [00:03<00:07,  7.03it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  7.02it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:06,  7.02it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  7.03it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  7.03it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  7.03it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  7.02it/s]\u001b[A\n",
            " 45% 36/80 [00:04<00:06,  7.03it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  7.03it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:05,  7.03it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  7.03it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  7.03it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  7.04it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  7.01it/s]\u001b[A\n",
            " 54% 43/80 [00:05<00:05,  7.02it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  7.02it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:04,  7.03it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  7.03it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  7.03it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  7.04it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  7.03it/s]\u001b[A\n",
            " 62% 50/80 [00:06<00:04,  7.03it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  7.03it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:03,  7.03it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  7.03it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  7.00it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  7.00it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  7.01it/s]\u001b[A\n",
            " 71% 57/80 [00:07<00:03,  7.02it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  7.02it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  7.03it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  7.03it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  7.03it/s]\u001b[A\n",
            " 80% 64/80 [00:08<00:02,  7.03it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  7.01it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:02,  6.99it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  7.00it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  7.01it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  7.01it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  7.02it/s]\u001b[A\n",
            " 89% 71/80 [00:09<00:01,  6.98it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  6.99it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:01,  6.96it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  6.98it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  7.00it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  7.00it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  7.01it/s]\u001b[A\n",
            " 98% 78/80 [00:10<00:00,  7.02it/s]\u001b[A\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 0.6999081969261169, 'eval_accuracy': 0.5, 'eval_runtime': 11.9563, 'eval_samples_per_second': 53.194, 'eval_steps_per_second': 6.691, 'epoch': 2.0}\n",
            " 67% 732/1098 [06:19<02:38,  2.31it/s]\n",
            "100% 80/80 [00:11<00:00,  7.02it/s]\u001b[A\n",
            "{'loss': 0.6925, 'learning_rate': 4.4626593806921675e-06, 'epoch': 2.73}\n",
            " 91% 1000/1098 [08:24<00:45,  2.14it/s]Saving model checkpoint to xlm-roberta-base_trainer/checkpoint-1000\n",
            "Configuration saved in xlm-roberta-base_trainer/checkpoint-1000/config.json\n",
            "Model weights saved in xlm-roberta-base_trainer/checkpoint-1000/pytorch_model.bin\n",
            "100% 1098/1098 [09:24<00:00,  2.32it/s]The following columns in the evaluation set don't have a corresponding argument in `XLMRobertaForSequenceClassification.forward` and have been ignored: index, text. If index, text are not expected by `XLMRobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 636\n",
            "  Batch size = 8\n",
            "\n",
            "  0% 0/80 [00:00<?, ?it/s]\u001b[A\n",
            "  2% 2/80 [00:00<00:05, 14.06it/s]\u001b[A\n",
            "  5% 4/80 [00:00<00:08,  8.84it/s]\u001b[A\n",
            "  6% 5/80 [00:00<00:09,  8.21it/s]\u001b[A\n",
            "  8% 6/80 [00:00<00:09,  7.82it/s]\u001b[A\n",
            "  9% 7/80 [00:00<00:09,  7.57it/s]\u001b[A\n",
            " 10% 8/80 [00:00<00:09,  7.40it/s]\u001b[A\n",
            " 11% 9/80 [00:01<00:09,  7.24it/s]\u001b[A\n",
            " 12% 10/80 [00:01<00:09,  7.17it/s]\u001b[A\n",
            " 14% 11/80 [00:01<00:09,  7.13it/s]\u001b[A\n",
            " 15% 12/80 [00:01<00:09,  7.10it/s]\u001b[A\n",
            " 16% 13/80 [00:01<00:09,  7.08it/s]\u001b[A\n",
            " 18% 14/80 [00:01<00:09,  7.07it/s]\u001b[A\n",
            " 19% 15/80 [00:01<00:09,  7.06it/s]\u001b[A\n",
            " 20% 16/80 [00:02<00:09,  7.05it/s]\u001b[A\n",
            " 21% 17/80 [00:02<00:08,  7.04it/s]\u001b[A\n",
            " 22% 18/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 24% 19/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 25% 20/80 [00:02<00:08,  7.03it/s]\u001b[A\n",
            " 26% 21/80 [00:02<00:08,  7.01it/s]\u001b[A\n",
            " 28% 22/80 [00:02<00:08,  7.02it/s]\u001b[A\n",
            " 29% 23/80 [00:03<00:08,  7.02it/s]\u001b[A\n",
            " 30% 24/80 [00:03<00:07,  7.02it/s]\u001b[A\n",
            " 31% 25/80 [00:03<00:07,  7.03it/s]\u001b[A\n",
            " 32% 26/80 [00:03<00:07,  7.03it/s]\u001b[A\n",
            " 34% 27/80 [00:03<00:07,  7.03it/s]\u001b[A\n",
            " 35% 28/80 [00:03<00:07,  7.00it/s]\u001b[A\n",
            " 36% 29/80 [00:03<00:07,  7.01it/s]\u001b[A\n",
            " 38% 30/80 [00:04<00:07,  7.02it/s]\u001b[A\n",
            " 39% 31/80 [00:04<00:06,  7.02it/s]\u001b[A\n",
            " 40% 32/80 [00:04<00:06,  7.02it/s]\u001b[A\n",
            " 41% 33/80 [00:04<00:06,  7.03it/s]\u001b[A\n",
            " 42% 34/80 [00:04<00:06,  7.03it/s]\u001b[A\n",
            " 44% 35/80 [00:04<00:06,  7.03it/s]\u001b[A\n",
            " 45% 36/80 [00:04<00:06,  7.03it/s]\u001b[A\n",
            " 46% 37/80 [00:05<00:06,  7.03it/s]\u001b[A\n",
            " 48% 38/80 [00:05<00:06,  6.99it/s]\u001b[A\n",
            " 49% 39/80 [00:05<00:05,  7.00it/s]\u001b[A\n",
            " 50% 40/80 [00:05<00:05,  6.98it/s]\u001b[A\n",
            " 51% 41/80 [00:05<00:05,  7.00it/s]\u001b[A\n",
            " 52% 42/80 [00:05<00:05,  7.01it/s]\u001b[A\n",
            " 54% 43/80 [00:05<00:05,  7.02it/s]\u001b[A\n",
            " 55% 44/80 [00:06<00:05,  7.02it/s]\u001b[A\n",
            " 56% 45/80 [00:06<00:04,  7.02it/s]\u001b[A\n",
            " 57% 46/80 [00:06<00:04,  7.03it/s]\u001b[A\n",
            " 59% 47/80 [00:06<00:04,  7.03it/s]\u001b[A\n",
            " 60% 48/80 [00:06<00:04,  7.03it/s]\u001b[A\n",
            " 61% 49/80 [00:06<00:04,  7.01it/s]\u001b[A\n",
            " 62% 50/80 [00:06<00:04,  7.02it/s]\u001b[A\n",
            " 64% 51/80 [00:07<00:04,  7.02it/s]\u001b[A\n",
            " 65% 52/80 [00:07<00:03,  7.01it/s]\u001b[A\n",
            " 66% 53/80 [00:07<00:03,  7.01it/s]\u001b[A\n",
            " 68% 54/80 [00:07<00:03,  7.02it/s]\u001b[A\n",
            " 69% 55/80 [00:07<00:03,  7.02it/s]\u001b[A\n",
            " 70% 56/80 [00:07<00:03,  7.00it/s]\u001b[A\n",
            " 71% 57/80 [00:07<00:03,  7.02it/s]\u001b[A\n",
            " 72% 58/80 [00:08<00:03,  7.02it/s]\u001b[A\n",
            " 74% 59/80 [00:08<00:03,  6.98it/s]\u001b[A\n",
            " 75% 60/80 [00:08<00:02,  6.99it/s]\u001b[A\n",
            " 76% 61/80 [00:08<00:02,  6.99it/s]\u001b[A\n",
            " 78% 62/80 [00:08<00:02,  7.00it/s]\u001b[A\n",
            " 79% 63/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 80% 64/80 [00:08<00:02,  7.02it/s]\u001b[A\n",
            " 81% 65/80 [00:09<00:02,  7.03it/s]\u001b[A\n",
            " 82% 66/80 [00:09<00:01,  7.03it/s]\u001b[A\n",
            " 84% 67/80 [00:09<00:01,  7.03it/s]\u001b[A\n",
            " 85% 68/80 [00:09<00:01,  7.03it/s]\u001b[A\n",
            " 86% 69/80 [00:09<00:01,  7.03it/s]\u001b[A\n",
            " 88% 70/80 [00:09<00:01,  7.03it/s]\u001b[A\n",
            " 89% 71/80 [00:09<00:01,  7.00it/s]\u001b[A\n",
            " 90% 72/80 [00:10<00:01,  7.01it/s]\u001b[A\n",
            " 91% 73/80 [00:10<00:00,  7.02it/s]\u001b[A\n",
            " 92% 74/80 [00:10<00:00,  7.02it/s]\u001b[A\n",
            " 94% 75/80 [00:10<00:00,  7.03it/s]\u001b[A\n",
            " 95% 76/80 [00:10<00:00,  7.03it/s]\u001b[A\n",
            " 96% 77/80 [00:10<00:00,  7.03it/s]\u001b[A\n",
            " 98% 78/80 [00:10<00:00,  7.03it/s]\u001b[A\n",
            "                                       \n",
            "\u001b[A{'eval_loss': 0.6971942186355591, 'eval_accuracy': 0.5, 'eval_runtime': 11.7444, 'eval_samples_per_second': 54.154, 'eval_steps_per_second': 6.812, 'epoch': 3.0}\n",
            "100% 1098/1098 [09:36<00:00,  2.32it/s]\n",
            "100% 80/80 [00:11<00:00,  7.03it/s]\u001b[A\n",
            "                                   \u001b[A\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 576.0446, 'train_samples_per_second': 15.233, 'train_steps_per_second': 1.906, 'train_loss': 0.6946410229514423, 'epoch': 3.0}\n",
            "100% 1098/1098 [09:36<00:00,  1.91it/s]\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "evalute_CTB.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}